
# Import libraries
import os
import pandas as pd
import numpy as np
import re
import json
import tweepy

import time
import pytz
from datetime import datetime
from langdetect import detect, lang_detect_exception
from nltk.corpus import stopwords

from nrclex import NRCLex
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from textblob import TextBlob

from wordcloud import WordCloud
import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')
import plotly.graph_objects as go
import plotly.express as px

# Set timezone
os.environ['TZ'] = 'Asia/Singapore'
time.tzset()


# Import data downloaded from Google Trends
df_google = pd.read_csv('google_377a.csv', skiprows=1)
df_google.columns = ['date', 'interest']


# Clean up data
def remove_inequalities(value): # Google Trends lists some values with '<'. We want to get rid of this to plot our graph.
    '''
    Removes the '<' inequality 
    '''
    if '<' in value:
        value = float(value[1]) - 1
    else:
        value = float(value)
    return value

df_google['interest'] = df_google['interest'].apply(lambda x: remove_inequalities(x))


# Create a figure        
fig = go.Figure()
fig.add_trace(go.Scatter(x=df_google['date'],
                         y=df_google['interest'],
                         mode='lines'))
fig.update_layout(title='Interest in 377A in Singapore by Google Search',
                  xaxis_title='Date',
                  yaxis_title='Interest',
                  autotypenumbers='convert types')


# Add vertical lines for important dates 
dates = ['2022-08-21', '2022-11-28']
for date in dates:
    fig.add_shape(type = "line",
                  x0=date, y0=0, x1=date, y1=100,
                  line = dict(color="red", dash='dot'))

fig.add_trace(go.Scatter(
    x=dates,
    y=[105, 105],
    text=['National Day Rally',
          'Parliament Debates'],
    mode="text"))

fig.update_layout(showlegend=False)
fig.show()


# Get Twitter API Credentials 
twitter_credentials = pd.read_csv('twitter_credentials.csv')
consumer_key = twitter_credentials['keys'][0]
consumer_secret = twitter_credentials['keys'][1]
access_token = twitter_credentials['keys'][2]
access_token_secret = twitter_credentials['keys'][3]


# Create the Authentication Object
authenticate = tweepy.OAuthHandler(consumer_key, consumer_secret)

# Set the Access Token
authenticate.set_access_token(access_token, access_token_secret)

# Create the API object while passing through authentication information
api = tweepy.API(authenticate, wait_on_rate_limit = True)


# Find out the tweet type 
def identify_tweet_type(tweet):
    '''
    Identifies whether a tweet is an original tweet, reply tweet, quote tweet or retweet.
    
    Parameters
    ----------
    tweet: Tweet object
        Tweet downloaded from the Twitter API using Tweepy
        
    Returns
    ----------
    tweet_type: str
        The type of the given tweet
    '''
    if tweet["in_reply_to_status_id"] is not None: # Check whether there is a reply
        tweet_type = "Reply Tweet"
    elif tweet["is_quote_status"] is True and not tweet["text"].startswith("RT"): # Check if it is a quote tweet and not a retweet of a quote tweet
        tweet_type = "Quote Tweet"
    elif tweet["text"].startswith("RT") and tweet.get("retweeted_status") is not None: # This checks if it is a retweet
        tweet_type = "Retweet"
    else:
        tweet_type = "Original Tweet"
    return tweet_type

# Process the tweets into a list of dicts
processed_tweets = []

def process_tweet(status):
    '''
    Processes tweets scraped from the Twitter API into a list of dict.
    
    Parameters
    ----------
    status: Status object
        Status obejct in Tweepy contains information about a tweet
        
    Returns
    ----------
    tweet_dict: list of dict
        Each dict represents a tweet and its attributes
    '''
    for tweet in status:
        if identify_tweet_type(tweet._json) == 'Retweet': # Get the full text of the retweet
            if 'extended_tweet' in tweet._json['retweeted_status']:
                full_text = tweet._json['retweeted_status']['extended_tweet']['full_text']
            else:
                full_text = tweet._json['retweeted_status']['text']

        else:
            if 'extended_tweet' in tweet._json: # Get the full text of the extended tweet
                full_text = tweet._json['extended_tweet']['full_text']
            else:
                full_text = tweet._json['text']

        tweet_dict = {"id": tweet._json["id_str"], # Only keep the relevant attributes
                  "date":tweet._json["created_at"],
                  "full_text": full_text,
                  "tweet_type": identify_tweet_type(tweet._json),
                  "hyperlink": "https://twitter.com/twitter/status/" + tweet._json["id_str"]
              }
        processed_tweets.append(tweet_dict)


# Scrape tweets from Twitter using Tweepy
for page in tweepy.Cursor(api.search_30_day, label='twitteranalysis', query="\"377A\"").pages(30):
    process_tweet(page)
tweets = pd.DataFrame(processed_tweets) # Save processed tweets into a Pandas DataFrame for easy cleaning and analysis later


# Remove duplicate tweets 
tweets = tweets.drop_duplicates(subset=['full_text'])

# Remove tweets in foreign languages
def find_lang(text):
    '''
    Identifies the language of a given text
    
    Parameters
    ----------
    text: str
        Original text
        
    Returns
    ----------
    lang: str 
        Language of text detected
    '''
    try:
        lang = detect(str(text))
    except lang_detect_exception.LangDetectException:
        lang = 'UNKNOWN'
    return lang

# Filter out tweets that are not in English
tweets['language'] = tweets['full_text'].apply(lambda x: find_lang(x)) # Add column to dataframe identifying language
tweets = tweets[tweets['language']=='en'] # Only keep tweets that are in English
tweets = tweets.drop('language', axis=1) # Remove the language column 

# Only keep tweets mentioning '377A' or '377a' exactly 
tweets = tweets[tweets['full_text'].str.contains('377A', case = False)]

# Reverse the order of the tweets so that they are in chronological order
tweets = tweets.iloc[::-1]

# Reset index
tweets = tweets.reset_index(drop=True)

# Check new df
tweets.head()


# Convert timezones from UTC to local time
def utc_to_local(date):
    '''
    Converts time from the UTC timezone to the local timezone.
    
    Parameters
    ----------
    date: str
        Date in the UTC timezone 
    
    Returns
    ----------
    date: datetime object
        Date in the local timezone
    '''
    date = datetime.strptime(date, '%a %b %d %H:%M:%S %z %Y')
    local_tz = datetime.now().astimezone().tzinfo
    date = date.replace(tzinfo=pytz.utc).astimezone(local_tz)
    return date
tweets['date'] = tweets['date'].apply(lambda x: utc_to_local(x))
tweets.head()


# Function to clean the text
def clean_text(text):
    '''
    Cleans the text for analysis by removing punctuation and symbols
    '''
    text = re.sub(r'@[A-Za-z0-9_]+', '', text) # Removes @mentions
    text = re.sub(r'#', '', text) # Removes the # symbol
    text = re.sub(r'RT[\s]+', '', text) # Removes RT/retweets
    text = re.sub(r'https?:\/\/\S+', '', text) # Removes hyperlinks
    text = re.sub(r'\n', ' ', text) # Removes new lines 
    text = re.sub(r'\\n', ' ', text) # Removes new lines
    text = re.sub(r'>', '', text) # Removes new lines
    return text

# Clean the tweets
tweets['full_text'] = tweets['full_text'].apply(lambda x: clean_text(x))
tweets.head()


# Import scraped Reddit data
singapore_1 = pd.read_csv('comments_singapore_p1.csv')
singapore_2 = pd.read_csv('comments_singapore_p2.csv')
singaporeraw_1 = pd.read_csv('comments_singaporeraw_p1.csv')
singaporeraw_2 = pd.read_csv('comments_singaporeraw_p2.csv')
reddit_comments = pd.concat([singapore_1, singapore_2, singaporeraw_1, singaporeraw_2], axis=0)


# Function for converting timezones from Unix time to local time in datetime object
def unix_to_local(time_recorded):
    '''
    Converts timezones from Unix time to local time
    
    Parameters
    ----------
    time_recorded : str
        Time recorded in Unix time (number of seconds that have elapsed since 00:00:00 UTC on 1 January 1970)
    
    Returns
    ----------
    time_recorded_datetime : datetime object
        Time recorded in local time in the format YYYY-MM-DD HH:MM:SS
    '''
    time_recorded_datetime = datetime.fromtimestamp(time_recorded)
    return time_recorded_datetime

# Clean reddit data
def clean_reddit(reddit_df):
    '''
    Cleans raw data downloaded from Reddit using `subreddit-comments-dl` for analysis
    '''
    reddit_df = reddit_df[reddit_df['body'].str.contains('377A', regex=False, case=False, na=False)] # Only keep comments mentioning '377A' or '377a'
    reddit_df = reddit_df.drop_duplicates(subset=['body']) # Drop any duplicates
    reddit_df = reddit_df.drop(['submission_id','parent_id'], axis=1) # Drop columns that are not required
    reddit_df.rename(columns = {'body':'full_text', 'created_utc': 'date'}, inplace=True)
    reddit_df = reddit_df[['id', 'date', 'full_text', 'subreddit', 'permalink']]
    reddit_df['full_text'] = reddit_df['full_text'].apply(lambda x: clean_text(x)) # Clean up text 
    reddit_df['date'] = reddit_df['date'].apply(lambda x: unix_to_local(x)) # Convert unix to local time in datetime object
    reddit_df.sort_values(by=['date'], inplace=True)
    reddit_df = reddit_df.reset_index(drop=True)
    return reddit_df

reddit_comments = clean_reddit(reddit_comments)
reddit_comments.head()


# Function for detecting subjectivity
def get_subjectivity(text):
    return TextBlob(text).sentiment.subjectivity

# Function for detecting polarity
def get_polarity(text):
    return TextBlob(text).sentiment.polarity


# Function for VADER analysis
def vader_analysis(text):
    sid = SentimentIntensityAnalyzer()
    sentiments = text.apply(sid.polarity_scores)
    return sentiments


# Function for detecting emotions
def get_emotions(text):
    text = NRCLex(text)
    return text.affect_frequencies


# Function for creating a DataFrame with sentiment analysis
def create_sentiment_df(text_df):
    '''
    Generates a DataFrame of sentiment analysis data for text, including emotion analysis, subjectivity, polarity and VADER analysis
    
    Parameters
    ----------
    text_df:
        Pandas DataFrame with data of text to be analysed
    
    Returns
    ----------
    sentiments_df:
        Pandas DataFrame with text and corresponding sentiment analysis
    '''
    # Adding emotion analysis from NRCLex 
    emotion_list = []
    for text in text_df['full_text']:
        emotion_list.append(get_emotions(text)) # We get a list of dict 
    sentiment_df = pd.DataFrame.from_dict(emotion_list)
    sentiment_df.insert(0,'full_text','') # Insert column with original full text
    sentiment_df['full_text'] = text_df['full_text']
    sentiment_df.insert(0,'date','') # Insert column with original date
    sentiment_df['date'] = text_df['date']
    sentiment_df = sentiment_df.drop(['anticip'], axis=1) # NRCLex sometimes creates a duplicate with an 'anticipation' and an 'anticip' column. We get rid of the 'anticip' column as all the values are '0.0'
    sentiment_df.replace(to_replace= np.nan, value = float(0), inplace = True)

    # Adding subjectivity 
    sentiment_df['subjectivity'] = sentiment_df.apply(lambda x: get_subjectivity(x['full_text']), axis=1)

    # Adding polarity
    sentiment_df['polarity'] = sentiment_df.apply(lambda x: get_polarity(x['full_text']), axis=1)

    # Adding VADER analysis to df
    sentiments = vader_analysis(text_df['full_text'])
    scores = ['neg', 'neu', 'pos', 'compound']
    for score in scores:
        sentiment_df['vader_' + score] = [sentiment[score] for sentiment in sentiments]

    # Reordering columns
    sentiment_df = sentiment_df[['full_text', 'date',
                                 'subjectivity', 'polarity',
                                 'fear', 'anger', 'sadness', 'disgust', 'negative',
                                 'trust', 'surprise', 'joy', 'anticipation', 'positive',
                                 'vader_neg', 'vader_neu', 'vader_pos', 'vader_compound']]

    return sentiment_df


# Generating sentiment analysis results for reddit comments
reddit_sentiments = create_sentiment_df(reddit_comments)
reddit_sentiments.head()


# Generating sentiment analysis results for tweets 
twitter_sentiments = create_sentiment_df(tweets)
twitter_sentiments.head()


# Finding Keywords 
stop_words = set(stopwords.words('english'))

def filter_words(sentiments):
    all_words = ' '.join([words for words in sentiments['full_text']]).split(' ')
    all_words = [word for word in all_words if word not in stop_words]
    all_words = [word for word in all_words if word.isalpha()]
    filtered_words = ' '.join(all_words)
    return filtered_words
twitter_words = filter_words(twitter_sentiments)
reddit_words = filter_words(reddit_sentiments)


# Plotting polarity and subjectivity
def polarity_subjectivity_plot(sentiments, color, title):
    '''
    Parameters:
    ----------
    sentiments (df)
    color (str)
    
    Returns
    ----------
    '''
    plt.figure(figsize=(8,6))
    for i in range(0, sentiments.shape[0]):
        plt.scatter(sentiments['polarity'][i], sentiments['subjectivity'][i], color=color)
    plt.title(title)
    plt.xlabel('Polarity')
    plt.ylabel('Subjectivity')
    return plt.show()


polarity_subjectivity_plot(twitter_sentiments, 'blue', 'Twitter Sentiments')


polarity_subjectivity_plot(reddit_sentiments, 'red', 'Reddit Sentiments')


# Plot boxplot for VADER compound scores
def plot_vader(sentiments_df, color, title):
    fig = go.Figure()
    fig.add_trace(go.Box(x=sentiments_df['vader_compound'],
                         line=dict(color=color),
                         boxmean=True,))
    fig.update_layout(title=title,
                      xaxis_title='VADER Compound Score',
                      yaxis_visible=False)
    return fig.show()


plot_vader(twitter_sentiments, 'blue', 'VADER Analysis of 377A Tweets')


plot_vader(reddit_sentiments, 'red', 'VADER Analysis of Reddit Comments')


# Conducting emotions analysis
def emotions_scatter_fig(sentiments, title):
    scatter_fig = px.scatter(sentiments,
                             x='date',
                             y=sentiments.columns[4:14],
                             trendline = 'ols',
                             title = title,
                             width=800,
                             height=800)
    return scatter_fig.show()


emotions_scatter_fig(twitter_sentiments, 'Emotions Analysis of Tweets on 377A')


emotions_scatter_fig(reddit_sentiments, 'Emotions Analysis of Reddit Comments on 377A')


# Plotting a word cloud with keywords
def plot_word_cloud(all_words):
    '''
    Plots a word cloud given a string of keywords
    '''
    word_cloud = WordCloud(width = 700, height = 500, random_state = 21, max_font_size = 120, collocations = False).generate(all_words)
    plt.imshow(word_cloud, interpolation = 'bilinear')
    plt.axis('off')
    return plt.show()


plot_word_cloud(twitter_words)


plot_word_cloud(reddit_words)

	id	date	full_text	tweet_type	hyperlink
0	1598126745726910464	Thu Dec 01 01:28:02 +0000 2022	@ChannelNewsAsia 377A keep for so many years t...	Reply Tweet	https://twitter.com/twitter/status/15981267457...
1	1598129259809472513	Thu Dec 01 01:38:02 +0000 2022	Small achievements for this month:\n1. Me bein...	Original Tweet	https://twitter.com/twitter/status/15981292598...
2	1598141674374692864	Thu Dec 01 02:27:21 +0000 2022	否极泰来 Piji Tailai: 377A is a beautification of ...	Original Tweet	https://twitter.com/twitter/status/15981416743...
3	1598142702566387712	Thu Dec 01 02:31:27 +0000 2022	@shuyeonify salty about 377a revoked ig	Reply Tweet	https://twitter.com/twitter/status/15981427025...
4	1598153715764318209	Thu Dec 01 03:15:12 +0000 2022	Of cos it is not. But for control freak, socie...	Original Tweet	https://twitter.com/twitter/status/15981537157...

	id	date	full_text	tweet_type	hyperlink
0	1598126745726910464	2022-12-01 09:28:02+08:00	@ChannelNewsAsia 377A keep for so many years t...	Reply Tweet	https://twitter.com/twitter/status/15981267457...
1	1598129259809472513	2022-12-01 09:38:02+08:00	Small achievements for this month:\n1. Me bein...	Original Tweet	https://twitter.com/twitter/status/15981292598...
2	1598141674374692864	2022-12-01 10:27:21+08:00	否极泰来 Piji Tailai: 377A is a beautification of ...	Original Tweet	https://twitter.com/twitter/status/15981416743...
3	1598142702566387712	2022-12-01 10:31:27+08:00	@shuyeonify salty about 377a revoked ig	Reply Tweet	https://twitter.com/twitter/status/15981427025...
4	1598153715764318209	2022-12-01 11:15:12+08:00	Of cos it is not. But for control freak, socie...	Original Tweet	https://twitter.com/twitter/status/15981537157...

	id	date	full_text	tweet_type	hyperlink
0	1598126745726910464	2022-12-01 09:28:02+08:00	377A keep for so many years to wait for singa...	Reply Tweet	https://twitter.com/twitter/status/15981267457...
1	1598129259809472513	2022-12-01 09:38:02+08:00	Small achievements for this month: 1. Me being...	Original Tweet	https://twitter.com/twitter/status/15981292598...
2	1598141674374692864	2022-12-01 10:27:21+08:00	否极泰来 Piji Tailai: 377A is a beautification of ...	Original Tweet	https://twitter.com/twitter/status/15981416743...
3	1598142702566387712	2022-12-01 10:31:27+08:00	salty about 377a revoked ig	Reply Tweet	https://twitter.com/twitter/status/15981427025...
4	1598153715764318209	2022-12-01 11:15:12+08:00	Of cos it is not. But for control freak, socie...	Original Tweet	https://twitter.com/twitter/status/15981537157...

	id	date	full_text	subreddit	permalink
0	il52jwx	2022-08-21 09:42:38	Singapore's religious leaders outline positi...	Singapore	/r/singapore/comments/wtnckb/singapores_religi...
1	il537nc	2022-08-21 09:48:01	Basically 377a repeal is coming and that relig...	Singapore	/r/singapore/comments/wtnckb/singapores_religi...
2	il53f7j	2022-08-21 09:49:44	Responding to questions about the matter, Arc...	Singapore	/r/singapore/comments/wtnckb/singapores_religi...
3	il546hb	2022-08-21 09:55:53	One of the leader of a Buddhist Group (Buddhis...	Singapore	/r/singapore/comments/wtnckb/singapores_religi...
4	il54htu	2022-08-21 09:58:25	Archbishop Goh, in the interview with The Stra...	Singapore	/r/singapore/comments/wtnckb/singapores_religi...

	full_text	date	subjectivity	polarity	fear	anger	sadness	disgust	negative	trust	surprise	joy	anticipation	positive	vader_neg	vader_neu	vader_pos	vader_compound
0	Singapore's religious leaders outline positi...	2022-08-21 09:42:38	0.327857	0.100679	0.043478	0.017391	0.034783	0.008696	0.060870	0.313043	0.017391	0.104348	0.165217	0.234783	0.020	0.884	0.096	0.9900
1	Basically 377a repeal is coming and that relig...	2022-08-21 09:48:01	0.471818	0.053636	0.000000	0.000000	0.000000	0.000000	0.125000	0.250000	0.000000	0.125000	0.250000	0.250000	0.000	0.942	0.058	0.4404
2	Responding to questions about the matter, Arc...	2022-08-21 09:49:44	0.395089	0.197768	0.061538	0.015385	0.030769	0.000000	0.076923	0.292308	0.030769	0.092308	0.169231	0.230769	0.019	0.863	0.118	0.9774
3	One of the leader of a Buddhist Group (Buddhis...	2022-08-21 09:55:53	0.133333	0.000000	0.000000	0.166667	0.000000	0.000000	0.000000	0.333333	0.000000	0.000000	0.000000	0.500000	0.051	0.899	0.051	0.0000
4	Archbishop Goh, in the interview with The Stra...	2022-08-21 09:58:25	0.497034	-0.170802	0.054795	0.041096	0.041096	0.000000	0.164384	0.150685	0.041096	0.095890	0.150685	0.260274	0.109	0.845	0.046	-0.9624

Sentiment Analysis on The Repeal of 377A in Singapore¶

Introduction¶

Scope¶

Data Description¶

Research Question¶

Time Period of Data¶

Data Scraping and Cleaning¶

Scraping Tweets¶

Cleaning Tweets¶

Scraping Reddit Comments¶

Cleaning Reddit Comments¶

Sentiment Analysis¶

Choice of NLP Packages¶

Generating Sentiment Analysis¶

Visualisations and Findings¶

Polarity and Subjectivity¶

Twitter¶

Reddit¶

VADER¶

Twitter¶

Reddit¶

Emotions Analysis¶

Twitter¶

Notable Emotions¶

Reddit¶

Notable Emotions¶

Word Cloud¶

Twitter¶

Reddit¶

Conclusion¶

1. Is the overall public sentiment towards the repeal of 377A positive or negative?¶

2. How polarised are opinions towards the repeal within Singapore society?¶

3. What specific emotions do people feel about the repeal?¶

4. How have sentiments towards the repeal changed over time?¶

Limitations and Areas for Improvement¶

Research Scope and Methodology¶

Data Science¶

References¶